import os
import gzip
import re
import time
import sys
import util
import logging
import model
import settings 
import cProfile
import pstats
import psyco
psyco.full()


def main():
    logging.basicConfig(filename='log', level=logging.INFO)
    Log = logging.getLogger('Main')

    main_clk = time.time()

    stat = model.Stat()
    db = model.Database()

    Log.info('.....................Start..........................\n')
    Log.info(str(time.ctime()))
    Log.info(settings.dirInput)

    # List file in DataTest (hostname == filename)
    for hostid , hostname in zip(range(1,settings.maximumHost),os.listdir(settings.dirInput)):
        start_t = time.time()
        # extract file from .gz 
        try:
            # read gzip
            start_t = time.time()
            gz = gzip.open(settings.dirInput + hostname ,'rb')
            content = gz.read()
            gz.close()
            hostname = 'http://'+hostname[:-3]+'/'
            Log.info('hostid %5d:\t%5.4f\t%s',hostid,time.time()-start_t,hostname)       #log

            # extract site and extract url
            start_t = time.time()
            tpage,thref = util.ExtractSites(hostname,content,db,stat)
            Log.info('\t\t%6.4f %10d pages %15d urls', time.time()-start_t, tpage,thref)  #log

            db.createtransposegraph()

            stat.tot_site += 1

        except IOError :
            Log.error('Cannot open %s',hostname)
            pass
    uniqurl = 0
    if db != None:
        uniqurl = db.totalurl
        #db.printgraph()
        db.close()

    stat.tot_hashurl = uniqurl
    stat.dump()
    #                                       Total site             Total page                Total url       
    Log.info('')
    Log.info('>> Summary')
    Log.info(str(time.ctime()))
    Log.info('Time usage     : %s',str(util.GetInHMS(int(time.time()-main_clk))))
    Log.info('Total sites    : %d sites',stat.tot_site)
    Log.info('Total pages    : %d pages',stat.tot_webpage)
    Log.info('Total href     : %d urls',stat.tot_hyperlink)
    Log.info('Total hash url : %d urls',uniqurl)
    #Log.info('Avg pages/site : %.4f',stat.tot_webpage*1.0/stat.tot_site)
    Log.info('......................END...........................\n')


if __name__ == "__main__" :
    try:
        #cProfile.run('main()')
        main()
        os.system('rm *.pyc')
    except KeyboardInterrupt:
        print 'End'
